{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"BERT.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true,"machine_shape":"hm","mount_file_id":"1fl6Abj5m0IWS3Oz9C_-ewU0Kz7PTQ2rL","authorship_tag":"ABX9TyPQzxL0WqeJ9DW1jj8nYc4f"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"markdown","metadata":{"id":"yi2wYNc0zJxM","colab_type":"text"},"source":["# 使用BERT进行中文情感分类\n","\n","\n"]},{"cell_type":"markdown","metadata":{"id":"ZeCgna2-zSbx","colab_type":"text"},"source":["# 一、数据获取"]},{"cell_type":"markdown","metadata":{"id":"CY7JXBONXJIE","colab_type":"text"},"source":["## 访问Google Drive上数据"]},{"cell_type":"code","metadata":{"id":"p3RUsqifzCI3","colab_type":"code","outputId":"e72af4fb-c7ba-41b9-d558-f3ee151089f0","executionInfo":{"status":"ok","timestamp":1584447406586,"user_tz":-480,"elapsed":1099,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":54}},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"8eFIpCFJ5jnT","colab_type":"code","colab":{}},"source":["import pandas as pd"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"llfJv5Q35EQa","colab_type":"code","colab":{}},"source":["path=\"/content/drive/My Drive/Colab Notebooks/NLP/04_BERT/\"\n","train=pd.read_csv(path+\"nCoV_100k_train.labled.csv\")\n","test=pd.read_csv(path+\"nCov_10k_test.csv\")"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fJ2jasbI9Kio","colab_type":"code","outputId":"ee65a9c3-ddf2-4282-b45d-1209012e0dd8","executionInfo":{"status":"ok","timestamp":1584447420319,"user_tz":-480,"elapsed":14812,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":442}},"source":["train.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>微博id</th>\n","      <th>微博发布时间</th>\n","      <th>发布人账号</th>\n","      <th>微博中文内容</th>\n","      <th>微博图片</th>\n","      <th>微博视频</th>\n","      <th>情感倾向</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>4456072029125500</td>\n","      <td>01月01日 23:50</td>\n","      <td>存曦1988</td>\n","      <td>写在年末冬初孩子流感的第五天，我们仍然没有忘记热情拥抱这2020年的第一天。带着一丝迷信，早...</td>\n","      <td>['https://ww2.sinaimg.cn/orj360/005VnA1zly1gah...</td>\n","      <td>[]</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>4456074167480980</td>\n","      <td>01月01日 23:58</td>\n","      <td>LunaKrys</td>\n","      <td>开年大模型…累到以为自己发烧了腰疼膝盖疼腿疼胳膊疼脖子疼#Luna的Krystallife#?</td>\n","      <td>[]</td>\n","      <td>[]</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>4456054253264520</td>\n","      <td>01月01日 22:39</td>\n","      <td>小王爷学辩论o_O</td>\n","      <td>�偳癯空饩褪俏业�，爹，发烧快好，毕竟美好的假期拿来养病不太好，假期还是要好好享受快乐，爹，...</td>\n","      <td>['https://ww2.sinaimg.cn/thumb150/006ymYXKgy1g...</td>\n","      <td>[]</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>4456061509126470</td>\n","      <td>01月01日 23:08</td>\n","      <td>芩鎟</td>\n","      <td>新年的第一天感冒又发烧的也太衰了但是我要想着明天一定会好的?</td>\n","      <td>['https://ww2.sinaimg.cn/orj360/005FL9LZgy1gah...</td>\n","      <td>[]</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>4455979322528190</td>\n","      <td>01月01日 17:42</td>\n","      <td>changlwj</td>\n","      <td>问：我们意念里有坏的想法了，天神就会给记下来，那如果有好的想法也会被记下来吗？答：那当然了。...</td>\n","      <td>[]</td>\n","      <td>[]</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["               微博id        微博发布时间  ... 微博视频 情感倾向\n","0  4456072029125500  01月01日 23:50  ...   []    0\n","1  4456074167480980  01月01日 23:58  ...   []   -1\n","2  4456054253264520  01月01日 22:39  ...   []    1\n","3  4456061509126470  01月01日 23:08  ...   []    1\n","4  4455979322528190  01月01日 17:42  ...   []    1\n","\n","[5 rows x 7 columns]"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"id":"KD3YhwX-JfB2","colab_type":"code","outputId":"7c88de0e-f410-4f6d-eaa5-2542f147ed6d","executionInfo":{"status":"ok","timestamp":1584447420320,"user_tz":-480,"elapsed":14801,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["columns=['id','date','user','content','image','vedio','target']\n","train.columns=columns\n","test.columns=columns[:-1]\n","train.drop(['id','date','user','image','vedio'],axis = 1,inplace = True)\n","train.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>content</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>写在年末冬初孩子流感的第五天，我们仍然没有忘记热情拥抱这2020年的第一天。带着一丝迷信，早...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>开年大模型…累到以为自己发烧了腰疼膝盖疼腿疼胳膊疼脖子疼#Luna的Krystallife#?</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>�偳癯空饩褪俏业�，爹，发烧快好，毕竟美好的假期拿来养病不太好，假期还是要好好享受快乐，爹，...</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>新年的第一天感冒又发烧的也太衰了但是我要想着明天一定会好的?</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>问：我们意念里有坏的想法了，天神就会给记下来，那如果有好的想法也会被记下来吗？答：那当然了。...</td>\n","      <td>1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                             content target\n","0  写在年末冬初孩子流感的第五天，我们仍然没有忘记热情拥抱这2020年的第一天。带着一丝迷信，早...      0\n","1    开年大模型…累到以为自己发烧了腰疼膝盖疼腿疼胳膊疼脖子疼#Luna的Krystallife#?     -1\n","2  �偳癯空饩褪俏业�，爹，发烧快好，毕竟美好的假期拿来养病不太好，假期还是要好好享受快乐，爹，...      1\n","3                     新年的第一天感冒又发烧的也太衰了但是我要想着明天一定会好的?      1\n","4  问：我们意念里有坏的想法了，天神就会给记下来，那如果有好的想法也会被记下来吗？答：那当然了。...      1"]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"markdown","metadata":{"id":"AXHS0TdaqrjW","colab_type":"text"},"source":["## 预处理数据"]},{"cell_type":"markdown","metadata":{"id":"7adThY3DVIfp","colab_type":"text"},"source":["### 去除空值"]},{"cell_type":"code","metadata":{"id":"2oeyBlPGUmkB","colab_type":"code","outputId":"bd66d470-35b0-4ad5-f6f5-a3e41fbc9114","executionInfo":{"status":"ok","timestamp":1584447420321,"user_tz":-480,"elapsed":14791,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":419}},"source":["train[train.isnull().T.any().T]  #找出含有nan的所有行"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>content</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>850</th>\n","      <td>NaN</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>1226</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1362</th>\n","      <td>NaN</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>2042</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2982</th>\n","      <td>NaN</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>97739</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>98243</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>98998</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>99055</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>99610</th>\n","      <td>NaN</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>434 rows × 2 columns</p>\n","</div>"],"text/plain":["      content target\n","850       NaN      1\n","1226      NaN      0\n","1362      NaN      1\n","2042      NaN      0\n","2982      NaN      1\n","...       ...    ...\n","97739     NaN      0\n","98243     NaN      0\n","98998     NaN      0\n","99055     NaN      0\n","99610     NaN      0\n","\n","[434 rows x 2 columns]"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"id":"GIiThdwvUy_u","colab_type":"code","colab":{}},"source":["train.dropna(axis=0, how='any', inplace=True)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"g4VPx56NjY8A","colab_type":"code","outputId":"efc4185f-fe05-4042-e10a-c7db8e0edbc4","executionInfo":{"status":"ok","timestamp":1584447420322,"user_tz":-480,"elapsed":14779,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":49}},"source":["train[train.isnull().T.any().T]  #找出含有nan的所有行"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>content</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["Empty DataFrame\n","Columns: [content, target]\n","Index: []"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"1lyHsrxOVO-y","colab_type":"code","outputId":"4377234c-3ef4-4cf1-d389-90b2c45e1eb6","executionInfo":{"status":"ok","timestamp":1584447420322,"user_tz":-480,"elapsed":14768,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["len(test[test.isnull().T.any().T])\n","#测试集37个为Nan值的数据"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["37"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"markdown","metadata":{"id":"-S8VL60wTNHp","colab_type":"text"},"source":["### 去除特殊符号"]},{"cell_type":"code","metadata":{"id":"ZTrOmH75TKTk","colab_type":"code","outputId":"5c687923-4846-4742-a77a-dd42c3436d08","executionInfo":{"status":"ok","timestamp":1584447420323,"user_tz":-480,"elapsed":14758,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["train['content'][15:25]"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["15    现在是北京时间11点38分我发了一个晚上的烧，难受死了，去给自己弄个冷毛巾吧，哇第一次发烧发...\n","16                        2020第一天就发烧，今年怕是可以过呢红红火火了。2芒市?\n","17                                       新年第一天在发烧当中过去了?\n","18                                        2020第一天就发烧很丧?\n","19    ??//@朴素善xi:是的，冷暖自知//@·飛行島嶼·:他们年少相识一起成长一起出道互相照顾...\n","20    可怜的大宝，今天咳嗽发烧，还被我训，被我打，为什么要乱扔东西呢？为什么要打人呢？妈妈骂你打你...\n","21    //@木堇LMWH:【??李汶翰??】???青春有你第一C位出道???全能ACE舞台王者??...\n","22                           打喷嚏咳嗽不捂住猪嘴的祝你发烧感冒到2030cnm?\n","23    晚上回家阿姨告知苗小苗发烧了，去了医院验血，嗓子和肺都没事，血象有点高，开了头孢。我们等候之...\n","24                   我太难了又喉咙痛，小发烧还要准备三个考试，还都是闭卷考，还都那么难?\n","Name: content, dtype: object"]},"metadata":{"tags":[]},"execution_count":10}]},{"cell_type":"code","metadata":{"id":"bK7QAkNFTDFK","colab_type":"code","colab":{}},"source":["import re\n","def clean(text):\n","    text = re.sub(r\"(回复)?(//)?\\s*@\\S*?\\s*(:| |$)\", \" \", text)  # 去除正文中的@和回复/转发中的用户名\n","    text = re.sub(r\"\\[\\S+\\]\", \"\", text)      # 去除表情符号\n","    # text = re.sub(r\"#\\S+#\", \"\", text)      # 保留话题内容\n","    URL_REGEX = re.compile(\n","        r'(?i)\\b((?:https?://|www\\d{0,3}[.]|[a-z0-9.\\-]+[.][a-z]{2,4}/)(?:[^\\s()<>]+|\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\))+(?:\\(([^\\s()<>]+|(\\([^\\s()<>]+\\)))*\\)|[^\\s`!()\\[\\]{};:\\'\".,<>?«»“”‘’]))',\n","        re.IGNORECASE)\n","    text = re.sub(URL_REGEX, \"\", text)       # 去除网址\n","    text = text.replace(\"转发微博\", \"\")       # 去除无意义的词语\n","    text = re.sub(r\"\\s+\", \" \", text) # 合并正文中过多的空格\n","    return text.strip()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"zLKoKbXQTbXl","colab_type":"code","colab":{}},"source":["train['content']=train['content'].apply(clean)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"o-SVrVznGzEr","colab_type":"code","outputId":"b04de4fc-2138-4765-9dcf-1711b53dcb51","executionInfo":{"status":"ok","timestamp":1584447422578,"user_tz":-480,"elapsed":16998,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["train['content'][15:25]"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["15    现在是北京时间11点38分我发了一个晚上的烧，难受死了，去给自己弄个冷毛巾吧，哇第一次发烧发...\n","16                        2020第一天就发烧，今年怕是可以过呢红红火火了。2芒市?\n","17                                       新年第一天在发烧当中过去了?\n","18                                        2020第一天就发烧很丧?\n","19    ?? 是的，冷暖自知 他们年少相识一起成长一起出道互相照顾经历了我看不到的岁月和曲折发烧感冒...\n","20    可怜的大宝，今天咳嗽发烧，还被我训，被我打，为什么要乱扔东西呢？为什么要打人呢？妈妈骂你打你...\n","21    【??李汶翰??】???青春有你第一C位出道???全能ACE舞台王者???李汶翰全能ACE?...\n","22                           打喷嚏咳嗽不捂住猪嘴的祝你发烧感冒到2030cnm?\n","23    晚上回家阿姨告知苗小苗发烧了，去了医院验血，嗓子和肺都没事，血象有点高，开了头孢。我们等候之...\n","24                   我太难了又喉咙痛，小发烧还要准备三个考试，还都是闭卷考，还都那么难?\n","Name: content, dtype: object"]},"metadata":{"tags":[]},"execution_count":13}]},{"cell_type":"code","metadata":{"id":"U208jOqmWDOO","colab_type":"code","outputId":"0269aa45-1a56-4e1e-fe1e-770f6ecd8a01","executionInfo":{"status":"ok","timestamp":1584447422579,"user_tz":-480,"elapsed":16981,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["len(train[train.isnull().T.any().T])"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"id":"SCotH28oWbi8","colab_type":"code","colab":{}},"source":["test.fillna(\"\", inplace = True)#以0填充"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"X-O3uHQ-Wfl4","colab_type":"code","colab":{}},"source":["test['content']=test['content'].apply(clean)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"DEOJQ9k0TPiO","colab_type":"text"},"source":["### 去除噪音标签"]},{"cell_type":"code","metadata":{"id":"_yhTNE4prNx2","colab_type":"code","outputId":"bab51043-85b9-47e8-cc76-101ee69567c3","executionInfo":{"status":"ok","timestamp":1584447422582,"user_tz":-480,"elapsed":16964,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":187}},"source":["train['target'].value_counts()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0     57287\n","1     25374\n","-1    16899\n","9         1\n","-         1\n","10        1\n","4         1\n","-2        1\n","·         1\n","Name: target, dtype: int64"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"KpUkFwIsuPnK","colab_type":"code","colab":{}},"source":["train_1=train.loc[(train['target']=='-1')] \n","train0=train.loc[(train['target']=='0')] \n","train1=train.loc[(train['target']=='1')] \n","train = pd.concat([train_1,train_1,train_1,train0,train1,train1])"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"cpLwVkvHw3tI","colab_type":"code","outputId":"4abbee95-13de-4f83-e068-9093de3faec6","executionInfo":{"status":"ok","timestamp":1584447422584,"user_tz":-480,"elapsed":16949,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":85}},"source":["train['target'].value_counts()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0     57287\n","1     50748\n","-1    50697\n","Name: target, dtype: int64"]},"metadata":{"tags":[]},"execution_count":19}]},{"cell_type":"code","metadata":{"id":"__9biYTxqt7m","colab_type":"code","outputId":"8d2e6c38-150d-4c1b-9afd-b42d370f26ee","executionInfo":{"status":"ok","timestamp":1584447422584,"user_tz":-480,"elapsed":16934,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["train.reset_index(drop=True,inplace=True)\n","train.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>content</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>开年大模型…累到以为自己发烧了腰疼膝盖疼腿疼胳膊疼脖子疼#Luna的Krystallife#?</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>发高烧反反复复，眼睛都快睁不开了。今天室友带我去看，还在发烧中建议我输液，我拒绝了。给我打针...</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>明天考试今天发烧跨年给我跨坏了？？2兰州·兰州交通大学?</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>我真的服了xkh昨天vv去和她说自己不舒服，描述了症状她说啊你这不是感冒没有发烧没事的晚上一...</td>\n","      <td>-1</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>一个人在外就对自己好一点吧，一个人过生日应该没什么尴尬的吧，今年的惊喜来自东哥2020第一天...</td>\n","      <td>-1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                             content target\n","0    开年大模型…累到以为自己发烧了腰疼膝盖疼腿疼胳膊疼脖子疼#Luna的Krystallife#?     -1\n","1  发高烧反反复复，眼睛都快睁不开了。今天室友带我去看，还在发烧中建议我输液，我拒绝了。给我打针...     -1\n","2                       明天考试今天发烧跨年给我跨坏了？？2兰州·兰州交通大学?     -1\n","3  我真的服了xkh昨天vv去和她说自己不舒服，描述了症状她说啊你这不是感冒没有发烧没事的晚上一...     -1\n","4  一个人在外就对自己好一点吧，一个人过生日应该没什么尴尬的吧，今年的惊喜来自东哥2020第一天...     -1"]},"metadata":{"tags":[]},"execution_count":20}]},{"cell_type":"code","metadata":{"id":"Ef3hTifZJRoh","colab_type":"code","outputId":"50eacdb7-4b09-433c-f352-999ae7eb372c","executionInfo":{"status":"ok","timestamp":1584447422585,"user_tz":-480,"elapsed":16922,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":204}},"source":["train=train.sample(frac=1)\n","train.head()"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>content</th>\n","      <th>target</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>90591</th>\n","      <td>太惨了也，今年不会还是我的本命年吧又发烧了已经退了，已经可以重新开心的玩耍了?</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>86153</th>\n","      <td>【#钟南山表示武汉还没有停止人传人#】钟南山：全国的确诊病例已经出现下降的趋势，目前武汉依然...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>82656</th>\n","      <td>转发。一定要注意打喷嚏的礼仪，喷嚏发出的气溶胶可以播散10米，“威力”巨大。随身携带纸巾，方...</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>124473</th>\n","      <td>卧槽许三多去钢七连了！！军队的钢刀！军人的剔骨刀！许三多！战斗吧！！！！好燃啊！?</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>14442</th>\n","      <td>闫某某和任某某两名同学擅自提前返校，并且返校后未及时向学院报告，属于瞒报个人信息行为。根据《...</td>\n","      <td>-1</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                                  content target\n","90591             太惨了也，今年不会还是我的本命年吧又发烧了已经退了，已经可以重新开心的玩耍了?      0\n","86153   【#钟南山表示武汉还没有停止人传人#】钟南山：全国的确诊病例已经出现下降的趋势，目前武汉依然...      0\n","82656   转发。一定要注意打喷嚏的礼仪，喷嚏发出的气溶胶可以播散10米，“威力”巨大。随身携带纸巾，方...      0\n","124473          卧槽许三多去钢七连了！！军队的钢刀！军人的剔骨刀！许三多！战斗吧！！！！好燃啊！?      1\n","14442   闫某某和任某某两名同学擅自提前返校，并且返校后未及时向学院报告，属于瞒报个人信息行为。根据《...     -1"]},"metadata":{"tags":[]},"execution_count":21}]},{"cell_type":"markdown","metadata":{"id":"mOmaotoJiCxu","colab_type":"text"},"source":["## 划分训练集和验证集"]},{"cell_type":"code","metadata":{"id":"Ro_aIxtviJL_","colab_type":"code","colab":{}},"source":["from sklearn.model_selection import train_test_split\n","X_train, X_val, y_train, y_val = train_test_split( train[\"content\"], train[\"target\"], test_size=0.1, random_state=42)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"auvNjljXLJII","colab_type":"text"},"source":["# 二、BERT 训练模型"]},{"cell_type":"code","metadata":{"id":"e8LByxo6LURX","colab_type":"code","outputId":"2f03ca36-6d1e-4142-b616-1ae150530c82","executionInfo":{"status":"ok","timestamp":1584447523863,"user_tz":-480,"elapsed":118181,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":938}},"source":["!pip install --upgrade tensorflow\n","import tensorflow as tf\n","from tensorflow.keras.layers import Dense, Input\n","from tensorflow.keras.optimizers import Adam\n","from tensorflow.keras.models import Model\n","from tensorflow.keras.callbacks import ModelCheckpoint\n","import tensorflow_hub as hub\n","import numpy as np"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting tensorflow\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/85/d4/c0cd1057b331bc38b65478302114194bd8e1b9c2bbc06e300935c0e93d90/tensorflow-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl (421.8MB)\n","\u001b[K     |████████████████████████████████| 421.8MB 33kB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.34.2)\n","Requirement already satisfied, skipping upgrade: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.0.8)\n","Requirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.1)\n","Collecting tensorboard<2.2.0,>=2.1.0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d9/41/bbf49b61370e4f4d245d4c6051dfb6db80cec672605c91b1652ac8cc3d38/tensorboard-2.1.1-py3-none-any.whl (3.8MB)\n","\u001b[K     |████████████████████████████████| 3.9MB 34.3MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.12.0)\n","Requirement already satisfied, skipping upgrade: numpy<2.0,>=1.16.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.18.1)\n","Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.8.1)\n","Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.10.0)\n","Requirement already satisfied, skipping upgrade: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.1.8)\n","Requirement already satisfied, skipping upgrade: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (3.2.0)\n","Requirement already satisfied, skipping upgrade: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.2.2)\n","Collecting tensorflow-estimator<2.2.0,>=2.1.0rc0\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/90/b77c328a1304437ab1310b463e533fa7689f4bfc41549593056d812fab8e/tensorflow_estimator-2.1.0-py2.py3-none-any.whl (448kB)\n","\u001b[K     |████████████████████████████████| 450kB 66.0MB/s \n","\u001b[?25hRequirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.0)\n","Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.24.3)\n","Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.1.0)\n","Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow) (0.9.0)\n","Requirement already satisfied, skipping upgrade: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow) (1.4.1)\n","Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow) (2.8.0)\n","Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (3.2.1)\n","Requirement already satisfied, skipping upgrade: requests<3,>=2.21.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (2.21.0)\n","Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (1.0.0)\n","Requirement already satisfied, skipping upgrade: setuptools>=41.0.0 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (45.2.0)\n","Requirement already satisfied, skipping upgrade: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (0.4.1)\n","Requirement already satisfied, skipping upgrade: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow) (1.7.2)\n","Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2019.11.28)\n","Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (2.8)\n","Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.0.4)\n","Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3,>=2.21.0->tensorboard<2.2.0,>=2.1.0->tensorflow) (1.24.3)\n","Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow) (1.3.0)\n","Requirement already satisfied, skipping upgrade: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (4.0)\n","Requirement already satisfied, skipping upgrade: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.1.1)\n","Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (0.2.8)\n","Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow) (3.1.0)\n","Requirement already satisfied, skipping upgrade: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow) (0.4.8)\n","\u001b[31mERROR: tensorflow-federated 0.12.0 has requirement tensorflow-addons~=0.7.0, but you'll have tensorflow-addons 0.8.3 which is incompatible.\u001b[0m\n","Installing collected packages: tensorboard, tensorflow-estimator, tensorflow\n","  Found existing installation: tensorboard 1.15.0\n","    Uninstalling tensorboard-1.15.0:\n","      Successfully uninstalled tensorboard-1.15.0\n","  Found existing installation: tensorflow-estimator 1.15.1\n","    Uninstalling tensorflow-estimator-1.15.1:\n","      Successfully uninstalled tensorflow-estimator-1.15.1\n","  Found existing installation: tensorflow 1.15.0\n","    Uninstalling tensorflow-1.15.0:\n","      Successfully uninstalled tensorflow-1.15.0\n","Successfully installed tensorboard-2.1.1 tensorflow-2.1.0 tensorflow-estimator-2.1.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"TUKCKmvNMaXu","colab_type":"code","outputId":"25eb881b-34a2-4d04-a466-364ba2a06a10","executionInfo":{"status":"ok","timestamp":1584447529955,"user_tz":-480,"elapsed":124260,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":122}},"source":["!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py\n","!pip install sentencepiece\n","import tokenization"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Collecting sentencepiece\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n","\r\u001b[K     |▎                               | 10kB 23.9MB/s eta 0:00:01\r\u001b[K     |▋                               | 20kB 3.0MB/s eta 0:00:01\r\u001b[K     |█                               | 30kB 4.4MB/s eta 0:00:01\r\u001b[K     |█▎                              | 40kB 2.8MB/s eta 0:00:01\r\u001b[K     |█▋                              | 51kB 3.5MB/s eta 0:00:01\r\u001b[K     |██                              | 61kB 4.1MB/s eta 0:00:01\r\u001b[K     |██▏                             | 71kB 4.8MB/s eta 0:00:01\r\u001b[K     |██▌                             | 81kB 5.4MB/s eta 0:00:01\r\u001b[K     |██▉                             | 92kB 6.0MB/s eta 0:00:01\r\u001b[K     |███▏                            | 102kB 4.7MB/s eta 0:00:01\r\u001b[K     |███▌                            | 112kB 4.7MB/s eta 0:00:01\r\u001b[K     |███▉                            | 122kB 4.7MB/s eta 0:00:01\r\u001b[K     |████                            | 133kB 4.7MB/s eta 0:00:01\r\u001b[K     |████▍                           | 143kB 4.7MB/s eta 0:00:01\r\u001b[K     |████▊                           | 153kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████                           | 163kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 174kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 184kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████                          | 194kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 204kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 215kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████                         | 225kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 235kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 245kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 256kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 266kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 276kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 286kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 296kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 307kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 317kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████                      | 327kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 337kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 348kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████                     | 358kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 368kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 378kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████                    | 389kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 399kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 409kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 419kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 430kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 440kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 450kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 460kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 471kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 481kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 491kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 501kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 512kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████                | 522kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 532kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 542kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 552kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 563kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 573kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 583kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 593kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 604kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 614kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 624kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 634kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 645kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 655kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 665kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 675kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████▏          | 686kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 696kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 706kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 716kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 727kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 737kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 747kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 757kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 768kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 778kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 788kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████▋       | 798kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 808kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 819kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 829kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 839kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 849kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 860kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 870kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 880kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 890kB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 901kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 911kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 921kB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 931kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 942kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▎  | 952kB 4.7MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▋  | 962kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 972kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 983kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 993kB 4.7MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 1.0MB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▏| 1.0MB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.0MB 4.7MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.0MB 4.7MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.0MB 4.7MB/s \n","\u001b[?25hInstalling collected packages: sentencepiece\n","Successfully installed sentencepiece-0.1.85\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"HsmRdUM0M3wE","colab_type":"text"},"source":["## 2.1 用于编码和构建模型的函数"]},{"cell_type":"markdown","metadata":{"id":"61Us-9qIjCH4","colab_type":"text"},"source":["编码函数将文本进行字编码、位置编码、片段编码"]},{"cell_type":"code","metadata":{"id":"Z9-EdtfINCQY","colab_type":"code","colab":{}},"source":["def bert_encode(texts, tokenizer, max_len=512):\n","    all_tokens = []\n","    all_masks = []\n","    all_segments = []\n","    print(texts)\n","    for text in texts:\n","        text = tokenizer.tokenize(text)\n","            \n","        text = text[:max_len-2]\n","        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n","        pad_len = max_len - len(input_sequence)\n","        \n","        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n","        tokens += [0] * pad_len\n","        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n","        segment_ids = [0] * max_len\n","        \n","        all_tokens.append(tokens)\n","        all_masks.append(pad_masks)\n","        all_segments.append(segment_ids)\n","    \n","    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"z4vwvjBWit5G","colab_type":"text"},"source":["构建模型，一个简单的baseline，经过BERT层之后，利用全连接层转到一个3分类的应用。"]},{"cell_type":"markdown","metadata":{"id":"Y3W0iZ1oaT7y","colab_type":"text"},"source":["自定义二分类的f1值函数,但keras是基于batch计算的，官方认为如果在batch基础上计算f1是具有误导性的，所以移除了f1等评价矩阵。"]},{"cell_type":"code","metadata":{"id":"GozHijHtPYZ3","colab_type":"code","outputId":"fa6df037-de51-458b-a5b5-d8159efecd98","executionInfo":{"status":"ok","timestamp":1584447530662,"user_tz":-480,"elapsed":124951,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["from keras import backend as K\n","\n","def f1(y_true, y_pred):\n","    def recall(y_true, y_pred):\n","        \"\"\"Recall metric.\n","\n","        Only computes a batch-wise average of recall.\n","\n","        Computes the recall, a metric for multi-label classification of\n","        how many relevant items are selected.\n","        \"\"\"\n","        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n","        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n","        recall = true_positives / (possible_positives + K.epsilon())\n","        return recall\n","\n","    def precision(y_true, y_pred):\n","        \"\"\"Precision metric.\n","\n","        Only computes a batch-wise average of precision.\n","\n","        Computes the precision, a metric for multi-label classification of\n","        how many selected items are relevant.\n","        \"\"\"\n","        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n","        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n","        precision = true_positives / (predicted_positives + K.epsilon())\n","        return precision\n","    precision = precision(y_true, y_pred)\n","    recall = recall(y_true, y_pred)\n","    return 2*((precision*recall)/(precision+recall+K.epsilon()))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Using TensorFlow backend.\n"],"name":"stderr"}]},{"cell_type":"markdown","metadata":{"id":"hK753Gh-cpfo","colab_type":"text"},"source":["在一次epoch结束之后计算相应的f1值(适合多分类)"]},{"cell_type":"code","metadata":{"id":"4FnUErfymDel","colab_type":"code","colab":{}},"source":["import numpy as np\n","from keras.callbacks import Callback\n","from sklearn.metrics import f1_score, precision_score, recall_score\n","class Metrics(Callback):\n","    def __init__(self, val_data):\n","        super().__init__()\n","        self.validation_data = val_data\n","        \n","    def on_train_begin(self, logs={}):\n","        #print(\"on train begin\", len(self.validation_data))\n","        self.val_f1s = []\n","        self.val_recalls = []\n","        self.val_precisions = []\n","\n","    def on_epoch_end(self, epoch, logs={}):\n","        #print(\"  on epoch end :\", len(self.validation_data))\n","        #进行预测\n","        xVal, yVal = self.validation_data\n","        val_pred = np.argmax(np.asarray(self.model.predict(xVal)), axis = -1)\n","        val_true = yVal\n","\n","        #计算f1值\n","        _val_f1 = f1_score(val_true, val_pred, average='macro')\n","        _val_recall = recall_score(val_true, val_pred, average='macro')\n","        _val_precision = precision_score(val_true, val_pred, average='macro')\n","        self.val_f1s.append(_val_f1)\n","        self.val_recalls.append(_val_recall)\n","        self.val_precisions.append(_val_precision)\n","        print('— val_f1: %f — val_precision: %f — val_recall %f' %(_val_f1, _val_precision, _val_recall))\n","        return"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"k7-yxISRc2MH","colab_type":"text"},"source":["搭建模型的方法"]},{"cell_type":"code","metadata":{"id":"4VlZU7uAM01P","colab_type":"code","colab":{}},"source":["def build_model(bert_layer, max_len=512):\n","    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n","    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n","    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n","\n","    _, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n","    clf_output = sequence_output[:, 0, :]\n","    out = Dense(3, activation='sigmoid')(clf_output)\n","    \n","    model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=out)\n","    model.compile(Adam(lr=2e-6), loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])\n","    \n","    return model"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"taT5H-axQWm6","colab_type":"text"},"source":["## 2.2 加载模型"]},{"cell_type":"markdown","metadata":{"id":"yy79DHFmkjXB","colab_type":"text"},"source":["从th-hub中加载一个BERT的预训练模型，并且将它作为Leras模型中的一层"]},{"cell_type":"code","metadata":{"id":"X6kiirxKQiB0","colab_type":"code","outputId":"eab645e8-ea31-4dd2-f58c-da78d82cd931","executionInfo":{"status":"ok","timestamp":1584447546510,"user_tz":-480,"elapsed":140780,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":51}},"source":["#load BERT as a KerasLayer\n","%time\n","#module_url = \"https://tfhub.dev/tensorflow/bert_zh_L-12_H-768_A-12/1\"\n","module_url=\"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1\"\n","bert_layer = hub.KerasLayer(module_url, trainable=True)\n","\n","#module_url=\"https://tfhub.dev/tensorflow/albert_en_xlarge/1\"\n","#bert_layer = hub.KerasLayer(module_url, trainable=True)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["CPU times: user 2 µs, sys: 1 µs, total: 3 µs\n","Wall time: 5.01 µs\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"tHAsFSOakxkA","colab_type":"text"},"source":["配置一个token工具，将根据BERT词表中的词把文本转化为一些列的token"]},{"cell_type":"code","metadata":{"id":"nIPOCZR7QqYK","colab_type":"code","colab":{}},"source":["#bert of tf-hub\n","vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n","do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\n","tokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n","\n","\n","#en albert of tf-hub\n","#sp_model_file = bert_layer.resolved_object.sp_model_file.asset_path.numpy()\n","#tokenizer = tokenization.FullSentencePieceTokenizer(sp_model_file)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"-ku0vxYTnQ0f","colab_type":"code","outputId":"390e2da8-c46a-4eb8-dfc5-087a2a80dc15","executionInfo":{"status":"ok","timestamp":1584447546515,"user_tz":-480,"elapsed":140768,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["tokenizer.tokenize(\"学习自然语言处理的BERT模型\")"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['学', '习', '自', '然', '语', '言', '处', '理', '的', 'BE', '##RT', '模', '型']"]},"metadata":{"tags":[]},"execution_count":31}]},{"cell_type":"markdown","metadata":{"id":"Y5bru4-plLuP","colab_type":"text"},"source":["## 2.3 训练模型"]},{"cell_type":"markdown","metadata":{"id":"NGKIncYQlaVJ","colab_type":"text"},"source":["实现文本的编码"]},{"cell_type":"code","metadata":{"id":"EUNeI4iDRAwE","colab_type":"code","outputId":"d095f7bb-413a-4476-8551-18a5a2e5aa80","executionInfo":{"status":"ok","timestamp":1584447621053,"user_tz":-480,"elapsed":215293,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":629}},"source":["train_input = bert_encode(X_train.astype(str), tokenizer, max_len=200)\n","val_input = bert_encode(X_val.astype(str), tokenizer, max_len=200)\n","test_input = bert_encode(test['content'].astype(str), tokenizer, max_len=200)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["39327     操他母亲了，打起仗来是不是还要看战斗机尾气符合不符合国5国6啊？我就不说其它国家的技术标准和...\n","17360                            我是真不明白那些瞒报行程瞒报发热的人脑子里是??吗?\n","84092     【武汉疫情背后的一条时间线】1.2019年12月30日，网上流传出一份盖有武汉卫健委公章的红...\n","4392      2020年新年快乐~今年是过的最人心惶惶的一个年，也希望新的病毒疫苗能尽快研发出来~希望一线...\n","110839    #抗疫一线群英谱##疫情当前警察不退#我是战士，我能坚持#战时政治建警# #抗击新型肺炎第一...\n","                                ...                        \n","40951     #请山西公布肺炎详细数据#请尽早公布运城有没有确诊和疑似的患者，听很多人说运城确诊了一例，还...\n","156791    #最新疫情地图#自己体温继续36.5正常，每天测试三次都正常。我因为得过胰腺炎，巴瑞特食管，...\n","156610    【国家卫健委：#全国疫情防控效果已经显现#】#滨海资讯#国家卫健委发言人米锋表示，截至2月1...\n","31071        大哥，叫你们下台不是因为你们封城啊，是因为你们渎职啊，懒政啊，不作为啊，你怎么这么拎不清呢？\n","3460      心情沉重也许这就是现实中的武汉吧，现在只能祈求一切顺利渡过，祈求病毒快点结束，祈求好友父母渡...\n","Name: content, Length: 142858, dtype: object\n","1312      李文亮不就是前车之鉴，呵呵。。 肯定不能进去！站着进去，跪着出来！ 派出所到底是什么地方，主...\n","65801     【官方回应较大数量发热病人未隔离收治：居家隔离的有20629人】湖北武汉仍有较大数量发热病人...\n","132330    #一封给2020的心愿单#国泰民安??????张艺兴我爱的人爱我的人身体健康事事顺心张艺兴四...\n","73                                        好烦躁。小宝发烧了2萍乡·芦溪县?\n","41319                         每天都在看古田有没有新增病例希望早点解封我想回家....?\n","                                ...                        \n","32680     看到大家都在骂川航我就放心了。航班调整不让免费退票客户端人工客服炸了电话客服也炸了如果我改签...\n","3069      前几天重庆机场到了500吨医疗物资，今天又到了200多吨医疗物资，如果我还听到医院说不够，重...\n","77007     武汉不明原因肺炎死亡患者为男性，61岁，因呼吸衰竭、重症肺炎入院，同时患有腹部肿瘤及慢性肝病...\n","105059    年年说复刻鬼知道会不会有按照苹果的尿性要是真出就直接上架官网按照当年se那价格也就3000多...\n","47034     看了一分钟看不下去了，新闻伦理是不该拍家人，但不说话，就没人帮他们。唉 “我儿子非常胖，能有...\n","Name: content, Length: 15874, dtype: object\n","0       #你好2020#新年第一天元气满满的早起出门买早饭结果高估了自己抗冻能力回家成功冻发烧（大概...\n","1       大宝又感冒鼻塞咳嗽了，还有发烧。队友加班几天不回。感觉自己的情绪在家已然是随时引爆的状态。情...\n","2                           还要去输两天液，这天也太容易感冒发烧了，一定要多喝热水啊?\n","3                                 我太难了别人怎么发烧都没事就我一检查甲型流感?\n","4       果然是要病一场的喽回来第三天开始感冒今儿还发烧了喉咙眼睛都难受的一匹怎么样能不经意让我的毕设...\n","                              ...                        \n","9995                              「2020的黑天鹅事件」>2019-nCov?\n","9996    �傂牧榧μ�#武汉加油#我们所有人，和我们这个国家一起，正在经历着一场这个星球上史无前例的考...\n","9997                          武大人民医院：发热咳嗽并非新冠肺炎的唯一首发症状(来自\n","9998                                闭关第二天发现一根白发2019-nCoV?\n","9999                    昨天还在想如果有动画短片就好了，今天就有了，视频果然更直接有效吧。\n","Name: content, Length: 10000, dtype: object\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"0WoYbWUMpuEJ","colab_type":"code","colab":{}},"source":["train_labels = train['target'].astype(int)+1"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"rw7vTzjRzgV3","colab_type":"code","colab":{}},"source":["y_train = y_train.astype(int)+1\n","y_val= y_val.astype(int)+1"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"JPc2Drr6RJw7","colab_type":"code","outputId":"62006c60-a398-404b-9757-60f7b859a915","executionInfo":{"status":"ok","timestamp":1584447621753,"user_tz":-480,"elapsed":215972,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":391}},"source":["model = build_model(bert_layer, max_len=200)\n","met=Metrics([val_input,y_val])\n","model.summary()"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Model: \"model\"\n","__________________________________________________________________________________________________\n","Layer (type)                    Output Shape         Param #     Connected to                     \n","==================================================================================================\n","input_word_ids (InputLayer)     [(None, 200)]        0                                            \n","__________________________________________________________________________________________________\n","input_mask (InputLayer)         [(None, 200)]        0                                            \n","__________________________________________________________________________________________________\n","segment_ids (InputLayer)        [(None, 200)]        0                                            \n","__________________________________________________________________________________________________\n","keras_layer (KerasLayer)        [(None, 768), (None, 177853441   input_word_ids[0][0]             \n","                                                                 input_mask[0][0]                 \n","                                                                 segment_ids[0][0]                \n","__________________________________________________________________________________________________\n","tf_op_layer_strided_slice (Tens [(None, 768)]        0           keras_layer[0][1]                \n","__________________________________________________________________________________________________\n","dense (Dense)                   (None, 3)            2307        tf_op_layer_strided_slice[0][0]  \n","==================================================================================================\n","Total params: 177,855,748\n","Trainable params: 177,855,747\n","Non-trainable params: 1\n","__________________________________________________________________________________________________\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pYB7_ILK01lY","colab_type":"code","colab":{}},"source":["a=[train_input[0][:100],train_input[1][:100],train_input[2][:100]]\n","validation_data=[train_input[0][100:150],train_input[1][100:150],train_input[2][100:150]]\n","validation_target= train_labels[100:150]     \n","validation_data2 = [validation_data,validation_target] "],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"GoBOEQg1kQRy","colab_type":"code","outputId":"cfcadb72-1962-4ef0-e69b-a7a8263600c3","executionInfo":{"status":"ok","timestamp":1584346090804,"user_tz":-480,"elapsed":9929226,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":309}},"source":["train_history = model.fit(\n","    train_input, y_train,\n","    epochs=5,\n","    batch_size=32,\n","    callbacks=[met]\n",")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Train on 142858 samples\n","Epoch 1/5\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.6930 - sparse_categorical_accuracy: 0.6964— val_f1: 0.729509 — val_precision: 0.728171 — val_recall 0.732841\n","142858/142858 [==============================] - 3522s 25ms/sample - loss: 0.6930 - sparse_categorical_accuracy: 0.6964\n","Epoch 2/5\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.6051 - sparse_categorical_accuracy: 0.7406— val_f1: 0.751982 — val_precision: 0.751073 — val_recall 0.754152\n","142858/142858 [==============================] - 3506s 25ms/sample - loss: 0.6051 - sparse_categorical_accuracy: 0.7406\n","Epoch 3/5\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.5562 - sparse_categorical_accuracy: 0.7630— val_f1: 0.755982 — val_precision: 0.762182 — val_recall 0.767436\n","142858/142858 [==============================] - 3505s 25ms/sample - loss: 0.5562 - sparse_categorical_accuracy: 0.7630\n","Epoch 4/5\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.5135 - sparse_categorical_accuracy: 0.7819— val_f1: 0.774503 — val_precision: 0.775366 — val_recall 0.782251\n","142858/142858 [==============================] - 3506s 25ms/sample - loss: 0.5136 - sparse_categorical_accuracy: 0.7819\n","Epoch 5/5\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.4731 - sparse_categorical_accuracy: 0.8006— val_f1: 0.791499 — val_precision: 0.790054 — val_recall 0.795747\n","142858/142858 [==============================] - 3506s 25ms/sample - loss: 0.4731 - sparse_categorical_accuracy: 0.8006\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"z-gGikXKztkj","colab_type":"code","colab":{}},"source":["model.save_weights(path+'bert_model_epoch5.h5')"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"zmQ0Rn_3RMgo","colab_type":"code","colab":{}},"source":["test_pred = model.predict(test_input)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"fh9MRsnRy1KT","colab_type":"code","outputId":"8fe5927b-ca9d-41e2-e250-0b1da36bef3a","executionInfo":{"status":"ok","timestamp":1584346786530,"user_tz":-480,"elapsed":1052,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["predictions = np.argmax(test_pred, axis=-1)-1\n","print(predictions)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[ 0 -1 -1 ...  0  0  1]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"KK6q1p370aqg","colab_type":"code","colab":{}},"source":["submit=pd.read_csv(path+'submit_example.csv')\n","submit['y']=predictions\n","submit.to_csv(path+'submission.csv', index=False)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"4OMV0K9RnI5u","colab_type":"text"},"source":["## 2.4 加载模型之后继续训练"]},{"cell_type":"code","metadata":{"id":"4k1Sf44ZnPrj","colab_type":"code","outputId":"ce90f5be-8f95-4853-84e7-576461f5fd5c","executionInfo":{"status":"ok","timestamp":1584360588909,"user_tz":-480,"elapsed":720545,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":207}},"source":["model_continue = build_model(bert_layer, max_len=200)\n","model_continue.load_weights(path+'bert_model_epoch8.h5')\n","train_history = model_continue.fit(\n","    train_input, y_train,\n","    epochs=3,\n","    batch_size=32,\n","    callbacks=[met]\n",")"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Train on 142858 samples\n","Epoch 1/3\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.4380 - sparse_categorical_accuracy: 0.8162— val_f1: 0.783824 — val_precision: 0.796388 — val_recall 0.798140\n","142858/142858 [==============================] - 3513s 25ms/sample - loss: 0.4380 - sparse_categorical_accuracy: 0.8162\n","Epoch 2/3\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.4049 - sparse_categorical_accuracy: 0.8317— val_f1: 0.816548 — val_precision: 0.816097 — val_recall 0.822694\n","142858/142858 [==============================] - 3506s 25ms/sample - loss: 0.4049 - sparse_categorical_accuracy: 0.8317\n","Epoch 3/3\n","142848/142858 [============================>.] - ETA: 0s - loss: 0.3762 - sparse_categorical_accuracy: 0.8447— val_f1: 0.825998 — val_precision: 0.824559 — val_recall 0.830033\n","142858/142858 [==============================] - 3506s 25ms/sample - loss: 0.3762 - sparse_categorical_accuracy: 0.8447\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zWBGPs8BQ3Rq","colab_type":"code","outputId":"aee9e11d-aad9-41f0-9b53-39ff3d92194e","executionInfo":{"status":"ok","timestamp":1584360963900,"user_tz":-480,"elapsed":80324,"user":{"displayName":"qingyuan liang","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhTJHoGFryhWrfj2D0X8Yu7JTP_jfz9n_P4els=s64","userId":"15260138906199842493"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"source":["model_continue.save_weights(path+'bert_model_epoch8.h5')\n","test_pred = model_continue.predict(test_input)\n","predictions = np.argmax(test_pred, axis=-1)-1\n","print(predictions)\n","submit=pd.read_csv(path+'submit_example.csv')\n","submit['y']=predictions\n","submit.to_csv(path+'submission.csv', index=False)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["[ 0 -1  0 ...  0  0  0]\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"B8Vt7VR6RtRR","colab_type":"text"},"source":["改进思路：  \n","删除文本中的人名（完成）  \n","将BERT模型换成ALBERT模型"]}]}